package com.syl.qianyin.spider.service;

import com.syl.framework.frametools.utils.RedisObjectUtils;
import com.syl.qianyin.spider.beans.IpProxyBean;
import com.syl.qianyin.spider.constants.RedisKeyConstant;
import com.syl.qianyin.spider.exend.interfaces.CustomPageProcessor;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.selector.Html;

import java.util.ListIterator;

/**
 * 西刺IP代理爬取
 *
 * @author syl
 * @create 2019-05-11 15:32
 **/
@Service
public class XiCiIPSpiderService implements CustomPageProcessor {
    private static final Logger LOG = LoggerFactory.getLogger(XiCiIPSpiderService.class);
    private static final String BASE_URL = "http://www.xicidaili.com/";
    private static final String BASE_PAGE_URL = BASE_URL + "nn/";

    @Autowired
    private RedisObjectUtils redisObjectUtils;
    @Autowired
    private RedisTemplate redisTemplate;
    @Value("${ip.poolSize}")
    private int poolSize;

    @Override
    public void start() {
        Long size = redisTemplate.opsForList().size(RedisKeyConstant.MAP_IP_POO_UNCONFIRMED);
        if(size > 0){
            LOG.info("ip 代理池(待验证) 还未更新 无需重新拉取");
            return;
        }
        String urlArray[] = new String[poolSize];
        for (int i = 0; i < urlArray.length; i++) {
            urlArray[i] = BASE_PAGE_URL + ( i + 1 );
        }
        LOG.info("开始添加ip 到ip代理池(待验证)");
        // 爬取ip 代理池不能使用多线程
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        Spider.create(this)
            .addUrl(urlArray)
            .setDownloader(httpClientDownloader)
            .addPipeline(new ConsolePipeline())
            .run();
    }

    public Site getSite() {
        Site site = CustomPageProcessor.super.getSite();
        site.setDomain(BASE_URL);
        return site;
    }

    @Override
    public void myProcess(Page page, String url, Html html, Document document) {
        ListIterator<Element> trList = document.select("#ip_list tr").listIterator();
        trList.next();// 跳过标题
        while (trList.hasNext()){
            Element next = trList.next();
            String ip = next.select("td:eq(1)").text();
            String port = next.select("td:eq(2)").text();
            String type = next.select("td:eq(5)").text();
            IpProxyBean ipBean = new IpProxyBean(ip, Integer.valueOf(port), type, url);
//            int size = next.select(".bar_inner.fast").size();
            int size = next.select(".bar_inner.slow").size();
            LOG.debug("ip连接状态是否良好  {} ip {} url {}", size, ip, url);
            if(size < 0)
                continue;
            LOG.debug("添加ip代理 {}", ipBean);
            redisObjectUtils.pushListCache(RedisKeyConstant.MAP_IP_POO_UNCONFIRMED, ipBean);
        }
    }

}
